{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "5_prepare_libsvm_data.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/GavinHacker/recsys_model/blob/master/5_prepare_libsvm_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "id": "yZzTzvPz99wr",
        "colab_type": "code",
        "outputId": "2453b1c8-77d7-447a-a7dd-5fee376a202e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "U0HLPpXzAB_-",
        "colab_type": "code",
        "outputId": "8b87b20f-b2cf-4e5a-84ea-e338c10f7d04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install pymysql"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pymysql\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ed/39/15045ae46f2a123019aa968dfcba0396c161c20f855f11dea6796bcaae95/PyMySQL-0.9.3-py2.py3-none-any.whl (47kB)\n",
            "\u001b[K    100% |████████████████████████████████| 51kB 2.0MB/s \n",
            "\u001b[?25hInstalling collected packages: pymysql\n",
            "Successfully installed pymysql-0.9.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "w2ai9wqbAMDc",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "g7x4QH7s_zIi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import pymysql\n",
        "import pymysql.cursors\n",
        "from functools import reduce\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import uuid\n",
        "import datetime\n",
        "#from pyfm import pylibfm\n",
        "from sklearn.feature_extraction import DictVectorizer\n",
        "from sklearn.metrics.pairwise import pairwise_distances\n",
        "np.set_printoptions(precision=3)\n",
        "np.set_printoptions(suppress=True)\n",
        "from sklearn.datasets import dump_svmlight_file\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "import pickle as pkl\n",
        "from sklearn import preprocessing"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "iHx13mBUpGf2",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 读取CSV到dataframe"
      ]
    },
    {
      "metadata": {
        "id": "OTcqErrS-8CM",
        "colab_type": "code",
        "outputId": "ab02e1d7-5f42-46e6-ffbf-389e9f4f6743",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "cell_type": "code",
      "source": [
        "df = pd.read_csv('/content/drive/My Drive/colabnotebook/comment_origin_data_1.csv',sep='\\t',encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/IPython/core/interactiveshell.py:2718: DtypeWarning: Columns (0,1,2,4,5,6,7,8,9,10,11,12,13,14,15,16,17,22) have mixed types. Specify dtype option on import or set low_memory=False.\n",
            "  interactivity=interactivity, compiler=compiler, result=result)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "Bhr9HwDh-4Az",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_data.to_csv('/content/drive/My Drive/colabnotebook/comment_origin_data.csv', sep='\\t', encoding='utf-8')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "m10beqzqATwN",
        "colab_type": "code",
        "outputId": "fcf5fa41-f4d0-46ec-d8f4-41f45c2f2ec9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(389689, 29)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "vNr0udIKAVLb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.drop_duplicates()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "O_2l-KdFB02s",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.drop(['Unnamed: 0'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UQ5HcXt6DJL_",
        "colab_type": "code",
        "outputId": "6738a160-ced4-4ae1-9622-3aaa5c6554ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df['ID'].shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289261,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "metadata": {
        "id": "uZzPuEBvFz0i",
        "colab_type": "code",
        "outputId": "83b7b67f-0c90-45e8-f233-799be1dd4b11",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289261, 28)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "metadata": {
        "id": "DUeP-5_cCxSu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.drop_duplicates(['ID'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v4onpWhiF7Gi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "0ToH1iCCGDhJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.head()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vSYjBAI8GgTT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.drop(['CONTENT'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "vdprvLaaGmxS",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3ccgxkjhGovi",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.drop(['ADD_TIME_x', 'ADD_TIME_y'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "b6YsB75OG_-5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "1SgvsDKHHBrZ",
        "colab_type": "code",
        "outputId": "71eb0d91-047c-44e7-b5b6-939bdd470478",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289228, 25)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "metadata": {
        "id": "GD8kI3lYHDQA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.iloc[10000]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "BqU3nZsjKFWw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.tail(10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fTzf7qWAKzPO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df = df.reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "WlwazXJnLMNb",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.tail(10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VHB1eN0QLRoK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df.head(1).T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VZ0ZuxtCMEwg",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main = df.drop(['name', 'CREATOR','description','img','ID','NEWDATA','MOVIEID'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "344kM6qMMhiB",
        "colab_type": "code",
        "outputId": "de170997-122a-4fdb-d9a4-2d4b20099ce1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289228, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "DJ-7LA0MMjC_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "st0zEq-xMkem",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ET5d9z5gNQFk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main.shape"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kXNLScDCNRMN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main = df_main.drop(['enable'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YsrCB_BhNS_j",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6Jiap_1PN8i9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#pd.to_datetime(df_main['TIME'], format='%d%b%Y:%H:%M:%S.%f')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "kBTk5h1TTyqw",
        "colab_type": "code",
        "outputId": "c38cf2ee-06a2-4cbf-bfde-4caf7b7744e0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "datetime.datetime.strptime(df['TIME'][0],'%Y-%m-%d %H:%M:%S').year - 2000"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "15"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "metadata": {
        "id": "toyHqFyKZ33o",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main = df_main.dropna(subset=['USERID','rcount']).reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8jImDpqbhnjw",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "mr0na9UfpOVu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 处理时间辅助函数"
      ]
    },
    {
      "metadata": {
        "id": "gcykyyCoX8ny",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "index = 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "i79kOPFqWrzA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def process_time(t):\n",
        "  global index\n",
        "  index = index + 1\n",
        "  try:\n",
        "    return datetime.datetime.strptime(t,'%Y-%m-%d %H:%M:%S').year - 2000\n",
        "  except:\n",
        "    print(index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "dDOUWBmOS7_Q",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main['TIME_DIS'] = df_main['TIME'].apply(lambda x: process_time(x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Eg9V_KcPYXxV",
        "colab_type": "code",
        "outputId": "421ac5ae-b371-4b91-835c-306b03d5a479",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.dropna(subset=['USERID']).shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289152, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "metadata": {
        "id": "aGLuXt-tSuGK",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "s8Am9BKMaNiJ",
        "colab_type": "code",
        "outputId": "f7779e55-5abc-4a00-d94a-b349c376725d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289152, 18)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "metadata": {
        "id": "wRbGpt9aaXhZ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main = df_main.drop(['TIME'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "RBhHUfbgajjB",
        "colab_type": "code",
        "outputId": "6002114a-b4e2-488c-c8df-2ed50b5a2d83",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289152, 17)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "metadata": {
        "id": "4oYzCImHgkRO",
        "colab_type": "code",
        "outputId": "bab62ef2-d7db-4343-fe17-0c15d0c32433",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.drop(['USERID'], axis=1).shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289152, 16)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "metadata": {
        "id": "GbCxHzhdam-0",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main.head().T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "619P1B4hiOM5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_connection():\n",
        "    return pymysql.connect(host='rm-2zeqqm6994abi7b6dqo.mysql.rds.aliyuncs.com',\n",
        "                               user='noone',\n",
        "                               password='Huawei12#$',\n",
        "                               db='recsys',\n",
        "                               port=3306,\n",
        "                               charset ='utf8',\n",
        "                               use_unicode=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8gfGRVb0iPFU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_movie = pd.read_sql_query(\"select * from movie\", get_connection())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "HxL0pGyfpcy9",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 获取各个维度数据的频度统计"
      ]
    },
    {
      "metadata": {
        "id": "fOWWo6qJhYQ-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def get_dim_dict(df, dim_name):\n",
        "  type_list = list(map(lambda x:x.split('|') ,df[dim_name]))\n",
        "  type_list = [x for l in type_list for x in l]\n",
        "  def reduce_func(x, y):\n",
        "    for i in x:\n",
        "      if i[0] == y[0][0]:\n",
        "        x.remove(i)\n",
        "        x.append(((i[0],i[1] + 1)))\n",
        "        return x\n",
        "    x.append(y[0])\n",
        "    return x\n",
        "  l = filter(lambda x:x != None, map(lambda x:[(x, 1)], type_list))\n",
        "  type_zip = reduce(reduce_func, list(l))\n",
        "  #type_list = sorted(list(set(type_list)))\n",
        "  #type_zip = zip(list(range(len(type_list))), type_list)\n",
        "  #print(len(type_zip))\n",
        "  #print(type_zip)\n",
        "  type_dict = {}\n",
        "  for i in type_zip:\n",
        "    type_dict[i[0]] = i[1]\n",
        "  return type_dict"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "JudeBkGWhcvA",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "type_dict = get_dim_dict(df_movie, 'type')\n",
        "actors_dict = get_dim_dict(df_movie, 'actors')\n",
        "director_dict = get_dim_dict(df_movie, 'director')\n",
        "trait_dict = get_dim_dict(df_movie, 'trait')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bxMQgKrdpl9_",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 把频度统计字典序列化，以备之后的模型使用"
      ]
    },
    {
      "metadata": {
        "id": "IfRfjyqDwvBn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "with open('/content/drive/My Drive/colabnotebook/actors_dict', 'wb') as f:\n",
        "  pkl.dump(actors_dict, f)\n",
        "  \n",
        "with open('/content/drive/My Drive/colabnotebook/director_dict', 'wb') as f:\n",
        "  pkl.dump(director_dict, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "OILpoTkHjFu3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_y = df_main['RATING']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oNhUWBhzjbww",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "df_main = df_main.drop(['RATING'], axis=1)\n",
        "df_main = df_main.drop(['userid'], axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "DE76Z9wRjlwI",
        "colab_type": "code",
        "outputId": "eea113b8-671f-4a11-acb8-8a47a64b8a7b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 514
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.head(1).T"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>USERID</th>\n",
              "      <td>cf2349f9c01f9a5cd4050aebd30ab74f</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>movieid</th>\n",
              "      <td>10533913</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>type</th>\n",
              "      <td>剧情|奇幻|冒险|喜剧</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>actors</th>\n",
              "      <td>艾米·波勒|菲利丝·史密斯|理查德·坎德|比尔·哈德尔|刘易斯·布莱克</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>region</th>\n",
              "      <td>美国</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>director</th>\n",
              "      <td>彼特·道格特|罗纳尔多·德尔·卡门</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>trait</th>\n",
              "      <td>感人|经典|励志</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rat</th>\n",
              "      <td>8.7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rmax</th>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rmin</th>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>ravg</th>\n",
              "      <td>3.85714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rcount</th>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rsum</th>\n",
              "      <td>27</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>rmedian</th>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>TIME_DIS</th>\n",
              "      <td>15</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            0\n",
              "USERID       cf2349f9c01f9a5cd4050aebd30ab74f\n",
              "movieid                              10533913\n",
              "type                              剧情|奇幻|冒险|喜剧\n",
              "actors    艾米·波勒|菲利丝·史密斯|理查德·坎德|比尔·哈德尔|刘易斯·布莱克\n",
              "region                                     美国\n",
              "director                    彼特·道格特|罗纳尔多·德尔·卡门\n",
              "trait                                感人|经典|励志\n",
              "rat                                       8.7\n",
              "rmax                                        5\n",
              "rmin                                        2\n",
              "ravg                                  3.85714\n",
              "rcount                                      7\n",
              "rsum                                       27\n",
              "rmedian                                     4\n",
              "TIME_DIS                                   15"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "metadata": {
        "id": "P96xHn-BjpqN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#df_main = df_main.drop([65416,45287,65414]).reset_index(drop=True)\n",
        "#df_main = df_main.drop([73432]).reset_index(drop=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lh49VTSoYNz8",
        "colab_type": "code",
        "outputId": "d91f3baf-a81f-4c6f-e5a6-dea1b9d6c354",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "df_main.shape, train_y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((289152, 15), (289152,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "metadata": {
        "id": "qTb3CB_gpq7c",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 构建训练样本的字典形式，准备之后One-hot编码和稀疏矩阵构建"
      ]
    },
    {
      "metadata": {
        "id": "-ILzscdAYngB",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "invalid_data_list = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "F6cbL4iQdJP3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "data_dict_list = []\n",
        "for i in df_main.index:\n",
        "  _dict = {}\n",
        "  is_invalid = False\n",
        "  #type\n",
        "  for s_type in df_main.iloc[i]['type'].split('|'):\n",
        "    _dict[s_type] = 1\n",
        "  #actors\n",
        "  for s_actor in df_main.iloc[i]['actors'].split('|'):\n",
        "    if not s_actor in actors_dict:\n",
        "      print('invalid data index is ' + str(i))\n",
        "      invalid_data_list.append(i)\n",
        "      is_invalid = True\n",
        "      break\n",
        "    if actors_dict[s_actor] < 2:\n",
        "      _dict['other_actor'] = 1\n",
        "    else:\n",
        "      _dict[s_actor] = 1\n",
        "  if is_invalid == True:\n",
        "    continue;\n",
        "  #regios\n",
        "  _dict[df_main.iloc[i]['region']] = 1\n",
        "  #userid ...\n",
        "  _dict[df_main.iloc[i]['USERID']] = 1\n",
        "  _dict[str(df_main.iloc[i]['movieid'])] = 1\n",
        "  _dict['rat'] = df_main.iloc[i]['rat']\n",
        "  _dict['rmax'] = df_main.iloc[i]['rmax']\n",
        "  _dict['rmin'] = df_main.iloc[i]['rmin']\n",
        "  _dict['ravg'] = df_main.iloc[i]['ravg']\n",
        "  _dict['rcount'] = df_main.iloc[i]['rcount']\n",
        "  _dict['rsum'] = df_main.iloc[i]['rsum']\n",
        "  _dict['rmedian'] = df_main.iloc[i]['rmedian']\n",
        "  _dict['TIME_DIS'] = df_main.iloc[i]['TIME_DIS']\n",
        "  #\n",
        "  #director\n",
        "  for s_director in df_main.iloc[i]['director'].split('|'):\n",
        "    if director_dict[s_director] < 2:\n",
        "      _dict['other_director'] = 1\n",
        "    else:\n",
        "      _dict[s_director] = 1\n",
        "  #trait\n",
        "  for s_trait in df_main.iloc[i]['trait'].split('|'):\n",
        "    _dict[s_trait] = 1\n",
        "  data_dict_list.append(_dict)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "_RQFKnueb-Qi",
        "colab_type": "code",
        "outputId": "094cf788-415e-4a72-d33c-1bdfca3a442a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "invalid_data_list"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 62
        }
      ]
    },
    {
      "metadata": {
        "id": "3rS-DCRP60zr",
        "colab_type": "code",
        "outputId": "688d932f-6dc5-4046-e45a-c297afa2c653",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "len(data_dict_list)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "289152"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        }
      ]
    },
    {
      "metadata": {
        "id": "tHa8EM6ZkAE9",
        "colab_type": "code",
        "outputId": "54055a6a-2b5c-4ece-8a67-f6c70944ba5d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "train_y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(289152,)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "metadata": {
        "id": "igU_R8yHqToD",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 使用DictVectorizer向量化"
      ]
    },
    {
      "metadata": {
        "id": "1PaWnzjai-t8",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "v = DictVectorizer()\n",
        "train_X = v.fit_transform(data_dict_list)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "7z7ipuvF7Vzf",
        "colab_type": "code",
        "outputId": "f6caaede-e5e1-4ca9-b0a1-4b04c96132fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "cell_type": "code",
      "source": [
        "v"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DictVectorizer(dtype=<class 'numpy.float64'>, separator='=', sort=True,\n",
              "        sparse=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "metadata": {
        "id": "V9kxKYjK-1GU",
        "colab_type": "code",
        "outputId": "a0a1f148-10ed-4903-c163-0507cf834046",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "train_X.shape, train_y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((289152, 75619), (289152,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 67
        }
      ]
    },
    {
      "metadata": {
        "id": "4MTZsHCS40gE",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#pd.DataFrame(data=train_X, columns=v.feature_names_).head(1).T"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "v2pvdEC3BY8s",
        "colab_type": "code",
        "outputId": "a3375865-2803-418a-d043-f8c00aea2fb4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "train_X.shape, train_y.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((289152, 75619), (289152,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 69
        }
      ]
    },
    {
      "metadata": {
        "id": "AjrjfDv-Oqpo",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# def fitpredict_libfm(trainX, trainY, testX, classification=True, rank=8, n_iter=100):\n",
        "#     encoder = OneHotEncoder(handle_unknown='ignore').fit(trainX)\n",
        "#     trainX = encoder.transform(trainX)\n",
        "#     testX = encoder.transform(testX)\n",
        "#     print(type(trainX))\n",
        "#     train_file = '/content/drive/My Drive/colabnotebook/libfm_train.txt'\n",
        "#     test_file = '/content/drive/My Drive/colabnotebook/libfm_test.txt'\n",
        "#     with open(train_file, 'w') as f:\n",
        "#         dump_svmlight_file(trainX, trainY, f=f)\n",
        "#     with open(test_file, 'w') as f:\n",
        "#         dump_svmlight_file(testX, numpy.zeros(testX.shape[0]), f=f)\n",
        "#     task = 'c' if classification else 'r'\n",
        "#     console_output = !$LIBFM_PATH -task $task -method mcmc -train $train_file -test $test_file -iter $n_iter -dim '1,1,$rank' -out output.libfm\n",
        "    \n",
        "#     libfm_pred = pandas.read_csv('output.libfm', header=None).values.flatten()\n",
        "#     return libfm_pred"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2QUEcOp5q0cD",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 拆分训练集和测试集"
      ]
    },
    {
      "metadata": {
        "id": "nr7IaqR0Pe-9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_X_ = train_X[0:280000]\n",
        "train_y_ = train_y[:280000]\n",
        "test_X_ = train_X[280000:]\n",
        "test_y_ = train_y[280000:]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8ESWT1AqcfFj",
        "colab_type": "code",
        "outputId": "b23bbf9d-b758-4a6f-b4a2-07302a898e97",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "train_X_.shape, train_y_.shape, test_X_.shape, test_y_.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((280000, 75619), (280000,), (9152, 75619), (9152,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "metadata": {
        "id": "1kmp4GRXccpc",
        "colab_type": "code",
        "outputId": "075decf6-09fc-44b9-cf07-c4208fe59a16",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "type(train_X_)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "scipy.sparse.csr.csr_matrix"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "metadata": {
        "id": "x7-_98KUPFgV",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "## save to libFM file"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "y8bP6KxOdGJU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train.txt'\n",
        "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test.txt'\n",
        "\n",
        "# dump_svmlight_file(train_X_, train_y_, train_file)\n",
        "# dump_svmlight_file(test_X_, test_y_, test_file)\n",
        "  \n",
        "  \n",
        "#   with open(test_file, 'w') as f:\n",
        "#       dump_svmlight_file(testX, numpy.zeros(testX.shape[0]), f=f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XKDaxHMkq5dC",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 构建逻辑斯蒂回归训练集和测试集"
      ]
    },
    {
      "metadata": {
        "id": "nP19V9yx-Vei",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "test_y_lr_ = test_y_.apply(lambda x: 1 if int(x) > 3 else 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XfaJSyRSeEfY",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_y_lr_ = train_y_.apply(lambda x: 1 if int(x) > 3 else 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AmpJ0MBPdnCF",
        "colab_type": "code",
        "outputId": "7cc3da71-67b0-46c4-de76-ec5f816726c9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "train_y_lr_.shape, test_y_lr_.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((280000,), (9152,))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 78
        }
      ]
    },
    {
      "metadata": {
        "id": "34UQeFEA1kAI",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "## save to lr file"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "y_JAmWaNeLI3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train_lr.txt'\n",
        "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test_lr.txt'\n",
        "\n",
        "# dump_svmlight_file(train_X_, train_y_lr_, train_file)\n",
        "# dump_svmlight_file(test_X_, test_y_lr_, test_file)\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ylmx9m7EykfF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "scaler = preprocessing.MaxAbsScaler()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "o2VTEnoiy0V1",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_X_scaling = scaler.fit_transform(train_X_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "81FptTTOzHOX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "test_X_scaling = scaler.fit_transform(test_X_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "rVU--bGpznna",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# save to scaling lr\n",
        "# train_file = '/content/drive/My Drive/colabnotebook/recsys_train_scaling_lr.txt'\n",
        "# test_file = '/content/drive/My Drive/colabnotebook/recsys_test_scaling_lr.txt'\n",
        "\n",
        "# dump_svmlight_file(train_X_scaling, train_y_lr_, train_file)\n",
        "# dump_svmlight_file(test_X_scaling, test_y_lr_, test_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "cabKj74urKuU",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 转换为libsvm数据，为之后libFM使用，LR也使用此类型数据"
      ]
    },
    {
      "metadata": {
        "id": "gqTIl8h-zadJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_file = '/content/drive/My Drive/colabnotebook/recsys_train_scaling.txt'\n",
        "test_file = '/content/drive/My Drive/colabnotebook/recsys_test_scaling.txt'\n",
        "\n",
        "dump_svmlight_file(train_X_scaling, train_y_, train_file)\n",
        "dump_svmlight_file(test_X_scaling, test_y_, test_file)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "xeMotHGCeXrX",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "with open('/content/drive/My Drive/colabnotebook/dict2vec', 'wb') as f:\n",
        "  pkl.dump(v, f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UNrvITmuhA7l",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "v_from_pkl = None\n",
        "with open('/content/drive/My Drive/colabnotebook/dict2vec', 'rb') as f:\n",
        "  v_from_pkl = pkl.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "bDUJ2FpJhJ-k",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "predict_x = v_from_pkl.transform({'美国':1})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "x0O5o-mYh9o_",
        "colab_type": "code",
        "outputId": "e0c87b57-1a4a-487b-ad4f-7b93fbe3c699",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "cell_type": "code",
      "source": [
        "predict_x.shape"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 75619)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 296
        }
      ]
    }
  ]
}